#!/bin/bash
#
# watch image server. 
# 
# If there is new build, upgrade test bed to latest build.
# This script should be run by corn every 5 or 10 minutes, to make sure test 
# bed is running latest build.
# 
# [Lock]
# There will be a lock file indicating which box is upgrading.
# 
# 1. When upgrading is under going, we need to create a lock file that indicates
# the box is unavailiable due to upgrading, and unlock after upgrade succeed.
# 2. If test is running, and we need to upgrade, upgrade will hold back 5 minutes
# at a time.
# 3. When lock file is empty, mean no box is occupied, we can do anything.
# 4. The lock file must exists, if the file is missing, stop all monitor and test,
# this means the system have bug, and need to be fixed ASAP, send out email if needed.
#

source include.sh

while [[ 1 ]]
do
    if [ -e "running.lck" ]; then
        echo "Test is running, wait for next time check until running.lck gone"
        sleep $test_delay
        continue
    fi
    echo -n "check whether image server is mounted.. "
    if [ ! -e "${build_local_point}/images" ]
    	then
    	echo "image server is not mounted, mount it now, you need to be root! Enter password if needed."
    	sudo mount -t cifs -o username=automation,password=password $img_server_addr $build_local_point
    	if [ $? -ne "0" ]
    		then
            echo "mount image server to local failed"
            exit 1
        fi
    fi
    echo "[ok]"
    build_branch=`cat buildbranch`
    build_path=${build_local_point}/images/${build_branch}/
    current_build_num=`cat cbuildnum`

    latest_build=`ls -t -l -r $build_path |awk '{print $NF}'|tail -n1`
    latest_build_num=${latest_build#Image*}
    echo "The latest build in ${build_branch} is ${latest_build_num}"
    
    if [ $latest_build_num -le $current_build_num ]
    	then
    	echo "Current build is latest, no need to upgrade."
        sleep $test_delay
        continue
    fi
    
    echo "There new build $latest_build_num in $build_branch, newer than \
stored $current_build_num, need to check if each DUT need to upgrade."
    
    get_models=(`grep 'model' $TOPOLOGY_FILE |sed -e '/^#/d; s/[\"\;]//g'|awk '{print $4}'|sed -e ':label;N;s/\n/ /;tlabel'`)
    #echo $get_models
    # upgrade test bed, and enable root
    for (( i = 0; i < ${#get_models[@]}; i++))
    do
        
    	model=${get_models[$i]}
        echo "--------------------------------------------------------"
    	echo $model
        echo "--------------------------------------------------------"
        dut_name=`echo $model| awk -F: '{print $1}'`
    	is_cluster=`echo $model| awk -F: '{print $2}'`
    	model=`echo $model| awk -F: '{print $3}'`
    	
        
        nick_name=`grep -i $model $MODELS_FILE|awk '{print $2}'`
        file_name=`grep -i $model $MODELS_FILE|sed -e 'N;s/\x0d//g'|awk '{print $3}'`
        
        img_path="${build_path}${latest_build}/${nick_name}/debug/${file_name}"
        
        if [ ! -e ${img_path}.ok ]
        	then
            echo "${img_path}.ok did not exists."
        	echo "Image file for $nick_name is not ready, need to check next time"
        	continue
        fi
    
        #echo "Image file is at $img_path"
    
        if [ "$is_cluster" != "fc" ]
        	then
        	#echo "Not cluster, find box's management IP.."
        	conn_ip=`grep ${dut_name}_mgmt_ip $TOPOLOGY_FILE|sed -e '/^#/d; s/[\"\;]//g'|awk '{print $4}'`
            
            box_build=`$WGCMD_STATUS $conn_ip status /system/version|xpath -q -e "/status/system/version/text()"`
            echo $box_build
            box_build=${box_build#*B*}
            
            if [ "$box_build" -eq "$latest_build_num" ]; then
                echo "Already latest build, no need to upgrade"
                #echo $box_build > ${conn_ip}.lck
                continue
            fi
            
            $UPGRADE_SINGLE $conn_ip $latest_build_num $img_path &

        else
        	#upgrade cluster!!
        	echo "cluster, need upgrade each members"
            conn_ip=`grep ${dut_name}_mgmt_ip $TOPOLOGY_FILE|sed -e '/^#/d; s/[\"\;]//g'|awk '{print $4}'`
            box_build=`$WGCMD_STATUS $conn_ip status /system/version|xpath -q -e "/status/system/version/text()"`
            box_build=${box_build#*B*}
            echo $box_build
        	master_conn_ip=`grep ${dut_name}_master_mip $TOPOLOGY_FILE|sed -e '/^#/d; s/[\"\;]//g'|awk '{print $4}'`
        	slave_conn_ip=`grep ${dut_name}_slave_mip $TOPOLOGY_FILE|sed -e '/^#/d; s/[\"\;]//g'|awk '{print $4}'`
            
            
            if [ "$box_build" -eq "$latest_build_num" ]; then
                echo "Already latest build, no need to upgrade"
                #echo $box_build > ${conn_ip}.lck
                continue
            fi
        	#echo $master_conn_ip
        	#echo $slave_conn_ip
            $UPGRADE_CLUSTER $conn_ip $master_conn_ip $slave_conn_ip $latest_build_num $img_path &
        fi
        echo ""
    done
    if [[ `ls *.lck 2>/dev/null|wc -l` -ge 1 ]]; then
        if [[ `grep -i fail *.lck|wc -l` -lt 1 ]]; then
            # no upgrade fail
            if [[ `cat *.lck|uniq|wc -l` -ne 1 ]]; then
                echo "Two dut have different build, wait for next time checking"
            elif [[ `cat *.lck|uniq` -eq $latest_build_num ]]; then
                echo "Two DUTs upgraded successfully, clear lock"
                # record current build number in cbuildnum and clear lock
                rm -fr *.lck
                echo $latest_build_num > cbuildnum
            fi
        else
            lock_name=`grep -i fail *.lck|awk -F: '{print $1}'`
            echo "$lock_name : upgrade failed, wait for next time upgrade"
            rm -fr $lock_name
        fi
    fi
    
    sleep $test_delay
done

